{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# coding: utf-8\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import os\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'\n",
    "\n",
    "import json\n",
    "import pickle\n",
    "import re\n",
    "import nltk\n",
    "from urllib.parse import unquote,urlparse\n",
    "import pandas as pd\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "import numpy\n",
    "from kafka import KafkaConsumer\n",
    "\n",
    "with open(\"SVM_model\",\"rb\")as f:\n",
    "    model=pickle.load(f)\n",
    "\n",
    "    \n",
    "with open(\"word2vec.pickle\",\"rb\") as f :\n",
    "    word2vec=pickle.load(f)\n",
    "    dictionary=word2vec[\"dictionary\"]\n",
    "    embeddings=word2vec[\"embeddings\"]\n",
    "    reverse_dictionary = word2vec[\"reverse_dictionary\"]\n",
    "\n",
    "\n",
    "def to_index(data):\n",
    "    d_index=[]\n",
    "    for word in data:\n",
    "        if word in dictionary.keys():\n",
    "            d_index.append(dictionary[word])\n",
    "        else:\n",
    "            d_index.append(dictionary[\"UNK\"])\n",
    "    return d_index\n",
    "\n",
    "\n",
    "def GeneSeg(payload):\n",
    "    #数字泛化为\"0\"\n",
    "    payload=payload.lower()\n",
    "    payload=unquote(unquote(payload))\n",
    "    payload,num=re.subn(r'\\d+',\"0\",payload)\n",
    "    #替换url为”http://u\n",
    "    payload,num=re.subn(r'(http|https)://[a-zA-Z0-9\\.@&/#!#\\?]+', \"http://u\", payload)\n",
    "    #分词\n",
    "    r = '''\n",
    "        (?x)[\\w\\.]+?\\(\n",
    "        |\\)\n",
    "        |\"\\w+?\"\n",
    "        |'\\w+?'\n",
    "        |http://\\w\n",
    "        |</\\w+>\n",
    "        |<\\w+>\n",
    "        |<\\w+\n",
    "        |\\w+=\n",
    "        |>\n",
    "        |[\\w\\.]+\n",
    "    '''\n",
    "\n",
    "    return nltk.regexp_tokenize(payload, r)\n",
    "\n",
    "def xss_detection(payload):\n",
    "    xssed_data=[]\n",
    "    maxlen=200\n",
    "    word=GeneSeg(payload)\n",
    "    xssed_data.append(word)\n",
    "    #print(xssed_data)\n",
    "    data=xssed_data[0]\n",
    "    #print(type(data))\n",
    "    datas_index=[to_index(data)]\n",
    "    datas_index=pad_sequences(datas_index,value=-1,maxlen=maxlen)\n",
    "    datas_embed=[]\n",
    "    dims=len(embeddings[\"UNK\"])\n",
    "    for datas_index in datas_index:\n",
    "        data_embed=[]\n",
    "        for d in datas_index:\n",
    "            if d != -1:\n",
    "                data_embed.extend(embeddings[reverse_dictionary[d]])\n",
    "            else:\n",
    "                data_embed.extend([0.0] * dims)\n",
    "        datas_embed.append(data_embed)\n",
    "\n",
    "    data=(datas_embed[0])\n",
    "    result=model.predict([data])\n",
    "    if result==numpy.array([1]):\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "\n",
    "    \n",
    "def xss(url):\n",
    "    parsed_url=urlparse(url)\n",
    "    query=parsed_url.query\n",
    "    #print(query)\n",
    "    if query=='':\n",
    "        return 0\n",
    "    else:\n",
    "        try:\n",
    "            variables = query.split('&')\n",
    "            values = [ variable.split('=')[1] for variable in variables if len(variable.split('='))>1 ]\n",
    "            for value in values:\n",
    "                result=xss_detection(value)\n",
    "                if result==True:\n",
    "                    return 1 #print(\"may be xss_attack:\",url)\n",
    "                else:\n",
    "                    return 0\n",
    "        except Exception as err:\n",
    "            return 0\n",
    "            #pass\n",
    "            #print(str(err))\n",
    "\n",
    "    #if linecount%1000==0:\n",
    "        #print(\"processed:\",linecount)\n",
    "\n",
    "\n",
    "xss('http://br-ofertasimperdiveis.epizy.com/produto.php?linkcompleto=iphone-6-plus-apple-64gb-cinza')\n",
    "xss('http://my-softbank-security.com/wap_login.htm')\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ConsumerRecord(topic='http', partition=0, offset=688, timestamp=-1, timestamp_type=0, key=None, value=b'{\"ts\":1538209508.209421,\"uid\":\"Cwufom1LRTgBSsNmv7\",\"id.orig_h\":\"192.168.1.190\",\"id.orig_p\":48590,\"id.resp_h\":\"220.181.112.244\",\"id.resp_p\":80,\"trans_depth\":1,\"method\":\"GET\",\"host\":\"www.baidu.com\",\"uri\":\"/index.html?aaa=<script>alert(1)</script>\",\"user_agent\":\"curl/7.59.0\",\"request_body_len\":0,\"response_body_len\":0,\"tags\":[]}', checksum=-929890586, serialized_key_size=-1, serialized_value_size=326)\n",
      "www.baidu.com/index.html?aaa=<script>alert(1)</script>  XSS \n",
      "www.baidu.com/index.html?aaa=<script>alert(1)</script>\n",
      "ConsumerRecord(topic='http', partition=0, offset=689, timestamp=-1, timestamp_type=0, key=None, value=b'{\"ts\":1538209513.931372,\"uid\":\"CNqZfH2Q9FYXd5fs33\",\"id.orig_h\":\"192.168.1.190\",\"id.orig_p\":45492,\"id.resp_h\":\"140.211.169.196\",\"id.resp_p\":80,\"trans_depth\":1,\"method\":\"GET\",\"host\":\"fedoraproject.org\",\"uri\":\"/static/hotspot.txt\",\"version\":\"1.1\",\"request_body_len\":0,\"response_body_len\":2,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"FgStqZ3n7qo9qtYJD3\"]}', checksum=1522004416, serialized_key_size=-1, serialized_value_size=369)\n",
      "fedoraproject.org/static/hotspot.txt\n",
      "ConsumerRecord(topic='http', partition=0, offset=690, timestamp=-1, timestamp_type=0, key=None, value=b'{\"ts\":1538209813.978381,\"uid\":\"CWFi0TGiBSrN0hRsi\",\"id.orig_h\":\"192.168.1.190\",\"id.orig_p\":39262,\"id.resp_h\":\"67.219.144.68\",\"id.resp_p\":80,\"trans_depth\":1,\"method\":\"GET\",\"host\":\"fedoraproject.org\",\"uri\":\"/static/hotspot.txt\",\"version\":\"1.1\",\"request_body_len\":0,\"response_body_len\":2,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"FjLinX289JALe9cMEi\"]}', checksum=1024021946, serialized_key_size=-1, serialized_value_size=366)\n",
      "fedoraproject.org/static/hotspot.txt\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-18-1a2af7ce32c2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mconsumer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mKafkaConsumer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'http'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mmessage\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mconsumer\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0;31m# message value and key are raw bytes -- decode if necessary!\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0;31m# e.g., for unicode: `message.value.decode('utf-8')`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/kafka/consumer/group.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1073\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_consumer_timeout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1074\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1075\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1076\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1077\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterator\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/kafka/consumer/group.py\u001b[0m in \u001b[0;36m_message_generator\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1011\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetcher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0min_flight_fetches\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1012\u001b[0m                 \u001b[0mpoll_ms\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1013\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpoll\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout_ms\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpoll_ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1014\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1015\u001b[0m             \u001b[0;31m# We need to make sure we at least keep up with scheduled tasks,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/kafka/client_async.py\u001b[0m in \u001b[0;36mpoll\u001b[0;34m(self, timeout_ms, future, delayed_tasks)\u001b[0m\n\u001b[1;32m    552\u001b[0m                 \u001b[0mtimeout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;36m1000.0\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# avoid negative timeouts\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    553\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 554\u001b[0;31m             \u001b[0mresponses\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_poll\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    555\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    556\u001b[0m             \u001b[0;31m# If all we had was a timeout (future is None) - only do one poll\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/kafka/client_async.py\u001b[0m in \u001b[0;36m_poll\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    566\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    567\u001b[0m         \u001b[0mstart_select\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 568\u001b[0;31m         \u001b[0mready\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_selector\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mselect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    569\u001b[0m         \u001b[0mend_select\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    570\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sensors\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib64/python3.6/selectors.py\u001b[0m in \u001b[0;36mselect\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    443\u001b[0m             \u001b[0mready\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    444\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 445\u001b[0;31m                 \u001b[0mfd_event_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_epoll\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpoll\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_ev\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    446\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mInterruptedError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    447\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mready\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "consumer = KafkaConsumer('http')\n",
    "for message in consumer:\n",
    "    print(message)\n",
    "    # message value and key are raw bytes -- decode if necessary!\n",
    "    # e.g., for unicode: `message.value.decode('utf-8')`\n",
    "    http_log = json.loads(message.value.decode('utf-8'))\n",
    "    http_query = http_log[\"host\"]+http_log[\"uri\"]\n",
    "    #print(http_query)\n",
    "    if xss(http_query):\n",
    "        print(http_query,\" XSS \")\n",
    "    \"\"\"\n",
    "    dns_query = http_log[\"query\"]\n",
    "    x = domain2list(extract(dns_query).domain)\n",
    "    X = mat([x,])\n",
    "    rs = results[model.predict_classes(X)[0][0]]\n",
    "    \n",
    "    print(rs, dns_query)\n",
    "    \"\"\"\n",
    "    print(http_query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
